{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I agree and disagree.  John is saying that the batters efforts will result\n",
      "in 4 more wins then losses.  While you are probably correct that 400%\n",
      "does not mean 4 more wins then losses, it means something.  I would\n",
      "rather have a player who increased my teams chances of winning by 1% in\n",
      "each of 400 PAs then I would a player who increased my chances of winning\n",
      "by .5% in each of 400 PAs.  Thus, there appears to me to be an obvious\n",
      "positive association between John's statistic and winning games.  Thus,\n",
      "before you disregard this stat, it appears to me that further study must\n",
      "go into what sort of relationship there is.\n"
     ]
    }
   ],
   "source": [
    "str = '''I agree and disagree.  John is saying that the batters efforts will result\n",
    "in 4 more wins then losses.  While you are probably correct that 400%\n",
    "does not mean 4 more wins then losses, it means something.  I would\n",
    "rather have a player who increased my teams chances of winning by 1% in\n",
    "each of 400 PAs then I would a player who increased my chances of winning\n",
    "by .5% in each of 400 PAs.  Thus, there appears to me to be an obvious\n",
    "positive association between John's statistic and winning games.  Thus,\n",
    "before you disregard this stat, it appears to me that further study must\n",
    "go into what sort of relationship there is.'''\n",
    "\n",
    "print str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I agree and disagree.',\n",
       " 'John is saying that the batters efforts will result\\nin 4 more wins then losses.',\n",
       " 'While you are probably correct that 400%\\ndoes not mean 4 more wins then losses, it means something.',\n",
       " 'I would\\nrather have a player who increased my teams chances of winning by 1% in\\neach of 400 PAs then I would a player who increased my chances of winning\\nby .5% in each of 400 PAs.',\n",
       " \"Thus, there appears to me to be an obvious\\npositive association between John's statistic and winning games.\",\n",
       " 'Thus,\\nbefore you disregard this stat, it appears to me that further study must\\ngo into what sort of relationship there is.']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "# 进行断句\n",
    "sens = nltk.sent_tokenize(str)\n",
    "\n",
    "sens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I',\n",
       " 'agree',\n",
       " 'and',\n",
       " 'disagree',\n",
       " '.',\n",
       " 'John',\n",
       " 'is',\n",
       " 'saying',\n",
       " 'that',\n",
       " 'the',\n",
       " 'batters',\n",
       " 'efforts',\n",
       " 'will',\n",
       " 'result',\n",
       " 'in',\n",
       " '4',\n",
       " 'more',\n",
       " 'wins',\n",
       " 'then',\n",
       " 'losses',\n",
       " '.',\n",
       " 'While',\n",
       " 'you',\n",
       " 'are',\n",
       " 'probably',\n",
       " 'correct',\n",
       " 'that',\n",
       " '400',\n",
       " '%',\n",
       " 'does',\n",
       " 'not',\n",
       " 'mean',\n",
       " '4',\n",
       " 'more',\n",
       " 'wins',\n",
       " 'then',\n",
       " 'losses',\n",
       " ',',\n",
       " 'it',\n",
       " 'means',\n",
       " 'something',\n",
       " '.',\n",
       " 'I',\n",
       " 'would',\n",
       " 'rather',\n",
       " 'have',\n",
       " 'a',\n",
       " 'player',\n",
       " 'who',\n",
       " 'increased',\n",
       " 'my',\n",
       " 'teams',\n",
       " 'chances',\n",
       " 'of',\n",
       " 'winning',\n",
       " 'by',\n",
       " '1',\n",
       " '%',\n",
       " 'in',\n",
       " 'each',\n",
       " 'of',\n",
       " '400',\n",
       " 'PAs',\n",
       " 'then',\n",
       " 'I',\n",
       " 'would',\n",
       " 'a',\n",
       " 'player',\n",
       " 'who',\n",
       " 'increased',\n",
       " 'my',\n",
       " 'chances',\n",
       " 'of',\n",
       " 'winning',\n",
       " 'by',\n",
       " '.5',\n",
       " '%',\n",
       " 'in',\n",
       " 'each',\n",
       " 'of',\n",
       " '400',\n",
       " 'PAs',\n",
       " '.',\n",
       " 'Thus',\n",
       " ',',\n",
       " 'there',\n",
       " 'appears',\n",
       " 'to',\n",
       " 'me',\n",
       " 'to',\n",
       " 'be',\n",
       " 'an',\n",
       " 'obvious',\n",
       " 'positive',\n",
       " 'association',\n",
       " 'between',\n",
       " 'John',\n",
       " \"'s\",\n",
       " 'statistic',\n",
       " 'and',\n",
       " 'winning',\n",
       " 'games',\n",
       " '.',\n",
       " 'Thus',\n",
       " ',',\n",
       " 'before',\n",
       " 'you',\n",
       " 'disregard',\n",
       " 'this',\n",
       " 'stat',\n",
       " ',',\n",
       " 'it',\n",
       " 'appears',\n",
       " 'to',\n",
       " 'me',\n",
       " 'that',\n",
       " 'further',\n",
       " 'study',\n",
       " 'must',\n",
       " 'go',\n",
       " 'into',\n",
       " 'what',\n",
       " 'sort',\n",
       " 'of',\n",
       " 'relationship',\n",
       " 'there',\n",
       " 'is',\n",
       " '.']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 进行分词\n",
    "words = nltk.word_tokenize(str)\n",
    "\n",
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I', 'PRP'),\n",
       " ('agree', 'VBP'),\n",
       " ('and', 'CC'),\n",
       " ('disagree', 'VBP'),\n",
       " ('.', '.'),\n",
       " ('John', 'NNP'),\n",
       " ('is', 'VBZ'),\n",
       " ('saying', 'VBG'),\n",
       " ('that', 'IN'),\n",
       " ('the', 'DT'),\n",
       " ('batters', 'NNS'),\n",
       " ('efforts', 'NNS'),\n",
       " ('will', 'MD'),\n",
       " ('result', 'VB'),\n",
       " ('in', 'IN'),\n",
       " ('4', 'CD'),\n",
       " ('more', 'JJR'),\n",
       " ('wins', 'NNS'),\n",
       " ('then', 'RB'),\n",
       " ('losses', 'NNS'),\n",
       " ('.', '.'),\n",
       " ('While', 'IN'),\n",
       " ('you', 'PRP'),\n",
       " ('are', 'VBP'),\n",
       " ('probably', 'RB'),\n",
       " ('correct', 'VBP'),\n",
       " ('that', 'IN'),\n",
       " ('400', 'CD'),\n",
       " ('%', 'NN'),\n",
       " ('does', 'VBZ'),\n",
       " ('not', 'RB'),\n",
       " ('mean', 'VB'),\n",
       " ('4', 'CD'),\n",
       " ('more', 'JJR'),\n",
       " ('wins', 'NNS'),\n",
       " ('then', 'RB'),\n",
       " ('losses', 'NNS'),\n",
       " (',', ','),\n",
       " ('it', 'PRP'),\n",
       " ('means', 'VBZ'),\n",
       " ('something', 'NN'),\n",
       " ('.', '.'),\n",
       " ('I', 'PRP'),\n",
       " ('would', 'MD'),\n",
       " ('rather', 'RB'),\n",
       " ('have', 'VB'),\n",
       " ('a', 'DT'),\n",
       " ('player', 'NN'),\n",
       " ('who', 'WP'),\n",
       " ('increased', 'VBD'),\n",
       " ('my', 'PRP$'),\n",
       " ('teams', 'NNS'),\n",
       " ('chances', 'NNS'),\n",
       " ('of', 'IN'),\n",
       " ('winning', 'VBG'),\n",
       " ('by', 'IN'),\n",
       " ('1', 'CD'),\n",
       " ('%', 'NN'),\n",
       " ('in', 'IN'),\n",
       " ('each', 'DT'),\n",
       " ('of', 'IN'),\n",
       " ('400', 'CD'),\n",
       " ('PAs', 'NNP'),\n",
       " ('then', 'RB'),\n",
       " ('I', 'PRP'),\n",
       " ('would', 'MD'),\n",
       " ('a', 'DT'),\n",
       " ('player', 'NN'),\n",
       " ('who', 'WP'),\n",
       " ('increased', 'VBD'),\n",
       " ('my', 'PRP$'),\n",
       " ('chances', 'NNS'),\n",
       " ('of', 'IN'),\n",
       " ('winning', 'VBG'),\n",
       " ('by', 'IN'),\n",
       " ('.5', 'NNP'),\n",
       " ('%', 'NN'),\n",
       " ('in', 'IN'),\n",
       " ('each', 'DT'),\n",
       " ('of', 'IN'),\n",
       " ('400', 'CD'),\n",
       " ('PAs', 'NNP'),\n",
       " ('.', '.'),\n",
       " ('Thus', 'RB'),\n",
       " (',', ','),\n",
       " ('there', 'EX'),\n",
       " ('appears', 'VBZ'),\n",
       " ('to', 'TO'),\n",
       " ('me', 'PRP'),\n",
       " ('to', 'TO'),\n",
       " ('be', 'VB'),\n",
       " ('an', 'DT'),\n",
       " ('obvious', 'JJ'),\n",
       " ('positive', 'JJ'),\n",
       " ('association', 'NN'),\n",
       " ('between', 'IN'),\n",
       " ('John', 'NNP'),\n",
       " (\"'s\", 'POS'),\n",
       " ('statistic', 'JJ'),\n",
       " ('and', 'CC'),\n",
       " ('winning', 'VBG'),\n",
       " ('games', 'NNS'),\n",
       " ('.', '.'),\n",
       " ('Thus', 'RB'),\n",
       " (',', ','),\n",
       " ('before', 'IN'),\n",
       " ('you', 'PRP'),\n",
       " ('disregard', 'VBP'),\n",
       " ('this', 'DT'),\n",
       " ('stat', 'NN'),\n",
       " (',', ','),\n",
       " ('it', 'PRP'),\n",
       " ('appears', 'VBZ'),\n",
       " ('to', 'TO'),\n",
       " ('me', 'PRP'),\n",
       " ('that', 'IN'),\n",
       " ('further', 'JJ'),\n",
       " ('study', 'NN'),\n",
       " ('must', 'MD'),\n",
       " ('go', 'VB'),\n",
       " ('into', 'IN'),\n",
       " ('what', 'WP'),\n",
       " ('sort', 'NN'),\n",
       " ('of', 'IN'),\n",
       " ('relationship', 'NN'),\n",
       " ('there', 'EX'),\n",
       " ('is', 'VBZ'),\n",
       " ('.', '.')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 进行词性标注\n",
    "tags = nltk.pos_tag(words)\n",
    "\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S\n",
      "  I/PRP\n",
      "  agree/VBP\n",
      "  and/CC\n",
      "  disagree/VBP\n",
      "  ./.\n",
      "  (PERSON John/NNP)\n",
      "  is/VBZ\n",
      "  saying/VBG\n",
      "  that/IN\n",
      "  the/DT\n",
      "  batters/NNS\n",
      "  efforts/NNS\n",
      "  will/MD\n",
      "  result/VB\n",
      "  in/IN\n",
      "  4/CD\n",
      "  more/JJR\n",
      "  wins/NNS\n",
      "  then/RB\n",
      "  losses/NNS\n",
      "  ./.\n",
      "  While/IN\n",
      "  you/PRP\n",
      "  are/VBP\n",
      "  probably/RB\n",
      "  correct/VBP\n",
      "  that/IN\n",
      "  400/CD\n",
      "  %/NN\n",
      "  does/VBZ\n",
      "  not/RB\n",
      "  mean/VB\n",
      "  4/CD\n",
      "  more/JJR\n",
      "  wins/NNS\n",
      "  then/RB\n",
      "  losses/NNS\n",
      "  ,/,\n",
      "  it/PRP\n",
      "  means/VBZ\n",
      "  something/NN\n",
      "  ./.\n",
      "  I/PRP\n",
      "  would/MD\n",
      "  rather/RB\n",
      "  have/VB\n",
      "  a/DT\n",
      "  player/NN\n",
      "  who/WP\n",
      "  increased/VBD\n",
      "  my/PRP$\n",
      "  teams/NNS\n",
      "  chances/NNS\n",
      "  of/IN\n",
      "  winning/VBG\n",
      "  by/IN\n",
      "  1/CD\n",
      "  %/NN\n",
      "  in/IN\n",
      "  each/DT\n",
      "  of/IN\n",
      "  400/CD\n",
      "  PAs/NNP\n",
      "  then/RB\n",
      "  I/PRP\n",
      "  would/MD\n",
      "  a/DT\n",
      "  player/NN\n",
      "  who/WP\n",
      "  increased/VBD\n",
      "  my/PRP$\n",
      "  chances/NNS\n",
      "  of/IN\n",
      "  winning/VBG\n",
      "  by/IN\n",
      "  .5/NNP\n",
      "  %/NN\n",
      "  in/IN\n",
      "  each/DT\n",
      "  of/IN\n",
      "  400/CD\n",
      "  PAs/NNP\n",
      "  ./.\n",
      "  Thus/RB\n",
      "  ,/,\n",
      "  there/EX\n",
      "  appears/VBZ\n",
      "  to/TO\n",
      "  me/PRP\n",
      "  to/TO\n",
      "  be/VB\n",
      "  an/DT\n",
      "  obvious/JJ\n",
      "  positive/JJ\n",
      "  association/NN\n",
      "  between/IN\n",
      "  (PERSON John/NNP)\n",
      "  's/POS\n",
      "  statistic/JJ\n",
      "  and/CC\n",
      "  winning/VBG\n",
      "  games/NNS\n",
      "  ./.\n",
      "  Thus/RB\n",
      "  ,/,\n",
      "  before/IN\n",
      "  you/PRP\n",
      "  disregard/VBP\n",
      "  this/DT\n",
      "  stat/NN\n",
      "  ,/,\n",
      "  it/PRP\n",
      "  appears/VBZ\n",
      "  to/TO\n",
      "  me/PRP\n",
      "  that/IN\n",
      "  further/JJ\n",
      "  study/NN\n",
      "  must/MD\n",
      "  go/VB\n",
      "  into/IN\n",
      "  what/WP\n",
      "  sort/NN\n",
      "  of/IN\n",
      "  relationship/NN\n",
      "  there/EX\n",
      "  is/VBZ\n",
      "  ./.)\n"
     ]
    }
   ],
   "source": [
    "# 进行命名实体识别\n",
    "ners = nltk.ne_chunk(tags)\n",
    "\n",
    "print ners"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 数据准备\n",
    "这里我们使用20news-bydate进行数据演示，可以从http://qwone.com/~jason/20Newsgroups/下载数据包，这个数据集并不算大，不过我们仍然会进行一些预处理。将分散的文件内容处理到如下JSON格式：\n",
    "```\n",
    "{'docid': value, 'topiclbl': value, 'content': value, 'test': value}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# files to create merged data\n",
    "DIRECTORY = '/Users/apple/Downloads/Temp/workspace/data/20news-bydate'\n",
    "TRAIN_RAW_DIR = DIRECTORY + '/20news-bydate-train/'\n",
    "TEST_RAW_DIR = DIRECTORY + '/20news-bydate-test/'\n",
    "TRAIN_OUT_FILE = DIRECTORY + '/merged_train.jsonl'\n",
    "TEST_OUT_FILE = DIRECTORY + '/merged_test.jsonl'\n",
    "\n",
    "import os\n",
    "import json\n",
    "import os\n",
    "import json\n",
    "import re\n",
    "\n",
    "def merge_tojsonl(srcdir, outfilename):\n",
    "\n",
    "    ''' merge all source files into one jsonl for nlp streaming\n",
    "    '''\n",
    "\n",
    "    rgx_newline = re.compile('\\n') # sub with ' '\n",
    "    bad_chars = bad_chars = '<>^'\n",
    "    rgx_general = re.compile('[%s]' % bad_chars) # sub with ''\n",
    "    \n",
    "    \n",
    "    def filter_filename(fname):\n",
    "        ''' util function to filter out unwanted files by filename '''\n",
    "        return fname[0] == '.'\n",
    "\n",
    "\n",
    "    def pre_process_doc(content):\n",
    "        ''' util function to sub for any troublesome chars '''\n",
    "        return rgx_general.sub('',rgx_newline.sub(' ', content))\n",
    "\n",
    "\n",
    "    def file_path_iter(source_dir):\n",
    "        ''' iterator over dir that generates full paths to files of interest '''\n",
    "        return (os.path.join(root, f)\n",
    "                for root, dirs, files in os.walk(source_dir)\n",
    "                for f in files if not filter_filename(f)\n",
    "               )\n",
    "\n",
    "\n",
    "    def write_to_file(path, dst):\n",
    "        ''' write dictionary of source document info to dst '''\n",
    "        with open(path, 'r', encoding='utf-8', errors='ignore') as src:\n",
    "            splitp = path.split('/')\n",
    "            json.dump({'docid': splitp[-1],\n",
    "                       'label': splitp[-2],\n",
    "                       'doc': pre_process_doc(src.read())\n",
    "                       }, dst)\n",
    "            dst.write('\\n')\n",
    "\n",
    "\n",
    "    ''' actually do stuff '''\n",
    "    with open(outfilename, 'w') as dst:\n",
    "        for path in file_path_iter(srcdir):\n",
    "            write_to_file(path, dst)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'utf8' codec can't decode byte 0xff in position 11883: invalid start byte",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-8ca3fcd6242a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmerge_tojsonl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTRAIN_RAW_DIR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTRAIN_OUT_FILE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mmerge_tojsonl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTEST_RAW_DIR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTEST_OUT_FILE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-16-5c44ffc45880>\u001b[0m in \u001b[0;36mmerge_tojsonl\u001b[0;34m(srcdir, outfilename)\u001b[0m\n\u001b[1;32m     54\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'w'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mdst\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfile_path_iter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrcdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m             \u001b[0mwrite_to_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdst\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-16-5c44ffc45880>\u001b[0m in \u001b[0;36mwrite_to_file\u001b[0;34m(path, dst)\u001b[0m\n\u001b[1;32m     47\u001b[0m                        \u001b[0;34m'label'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0msplitp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m                        \u001b[0;34m'doc'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mpre_process_doc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m                        }, dst)\n\u001b[0m\u001b[1;32m     50\u001b[0m             \u001b[0mdst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/apple/anaconda/envs/word2vec/lib/python2.7/json/__init__.pyc\u001b[0m in \u001b[0;36mdump\u001b[0;34m(obj, fp, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, encoding, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;31m# could accelerate with writelines in some versions of Python, at\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m     \u001b[0;31m# a debuggability cost\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 189\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    190\u001b[0m         \u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    191\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/apple/anaconda/envs/word2vec/lib/python2.7/json/encoder.pyc\u001b[0m in \u001b[0;36m_iterencode\u001b[0;34m(o, _current_indent_level)\u001b[0m\n\u001b[1;32m    432\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 434\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_iterencode_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_current_indent_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    435\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mchunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    436\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/apple/anaconda/envs/word2vec/lib/python2.7/json/encoder.pyc\u001b[0m in \u001b[0;36m_iterencode_dict\u001b[0;34m(dct, _current_indent_level)\u001b[0m\n\u001b[1;32m    388\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0m_key_separator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbasestring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 390\u001b[0;31m                 \u001b[0;32myield\u001b[0m \u001b[0m_encoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    391\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    392\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0;34m'null'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'utf8' codec can't decode byte 0xff in position 11883: invalid start byte"
     ]
    }
   ],
   "source": [
    "merge_tojsonl(TRAIN_RAW_DIR, TRAIN_OUT_FILE)\n",
    "merge_tojsonl(TEST_RAW_DIR, TEST_OUT_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
